"""
DpipweSource is the feeder for sensor data provided by dpipwe.
This feeder iterates through each valid file specified by the
path provided by the delegate configuration, parses those files and 
dispatches all valid samples. 


@author Ben Howell <ben.howell@csiro.au>
"""

import time
import sys
import os
import xml

import string
import threading
from threading import Thread, Event, InterruptedException
from decimal import Decimal

from java.lang import Boolean

from com.xhaus.jyson import JysonCodec as json

from org.joda.time import DateTimeZone
from org.joda.time.format import DateTimeFormat 


from au.csiro.ict.tasman.core import Message
from au.csiro.ict.tasman.util import QuickCache
from au.csiro.ict.tasman.datatype import Sample
from sleeper import Sleeper
from simplexlsx import SimpleXlsxParser
from simplexlsx import SimpleXlsxDate as xl_date
from zlibextract import ZlibExtract as zlib_extract



def run():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  This is the entry point for this script. This call returns the running script
  instance which will, on subsequent calls be held by the "instance" parameter.
  An almost, sort of visitor pattern if you like.
  @return the running script instance.
  """
  return DpipweSource()
  

def isRunning():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  If false, SourceManager will clean up calling thread and attempt a restart on
  the Java side.
  @return true if running, else false.
  """
  if instance is not None:
    if instance.is_running():
      return Boolean("True")
  return Boolean("False")


def shutDown():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  This allows script to perform its own cleanup routine before exiting.
  @return true after shutdown.
  """
  if instance is not None:
    if instance.shutdown():
      return Boolean("True")
  return Boolean("False")
    


class DpipweSource:
  """
  Main data source routine.
  """
  def __init__(self):
    """
    Constructor.
    """
    import re
    self.sleeper = Sleeper()
    self.thread_cancelled = False
    self.configuration = json.loads(source.getConfiguration().getConfiguration().toString())
    self.file_name_pattern = re.compile(self.configuration['filenamepattern'])
    self.interval = self.configuration['interval']
    
    self.metadata_dir = source.getDataDirectory() + self.configuration['metadatapath']
    self.quality_code_archive = self.metadata_dir + self.configuration['qualitycodes']
    self.data_dir = source.getDataDirectory() + self.configuration['datapath']
    self.cache = QuickCache(self.data_dir + "cache.json")
    
    self.thread = Thread(target=self.run)
    self.thread.start()
    source.getLogger().info("DpipweSource initialised")
    
    
  def run(self):
    """
    Main loop.
    """
    sxlp = SimpleXlsxParser()
    dsxh = DpipweSourceXlsxHandler()
    tsxh = TimeStudioManagerQualityCodesXlsxHandler()

    # Extract quality codes.
    source.getLogger().debug("parsing quality code workbook: " + self.quality_code_archive)
    sxlp.set_content_handler(tsxh)
    sxlp.parse_workbook(self.quality_code_archive)
    dsxh.quality_code_dictionary = tsxh.quality_code_dictionary
    sxlp.set_content_handler(dsxh)
    
    while not self.thread_cancelled:
      try:
	for file_name in os.listdir(self.data_dir):
	  match = self.file_name_pattern.match(file_name)
	  if match and not self.cache.contains(file_name):
	    self.cache.add(file_name)
	    workbook = os.path.join(self.data_dir, file_name)
	    source.getLogger().debug("parsing workbook: " + workbook)
	    sxlp.parse_workbook(workbook)
	self.cache.save()
	source.getLogger().debug("sleeping...")
	self.sleeper.sleep(self.interval)
	source.getLogger().debug("waking...")
      except InterruptedException:
	self.thread_cancelled = True
	
    
  def is_running(self):
    """
    Returns true if main loop is running.
    @return true if main loop is running.
    """
    return self.thread.isAlive()
    
  
  def shutdown(self):
    """
    Performs cleanup routine on shutdown.
    @return true after shutdown.
    """
    source.getLogger().debug("shutting down script...")
    self.thread_cancelled = True
    source.getLogger().debug("forcibly waking thread...")
    self.sleeper.waken()
    #block while waiting for thread to terminate
    while self.thread.isAlive():
      time.sleep(1)
    source.getLogger().debug("script terminated!")
    self.cache.save()
    return True
    
    
    
class TimeStudioManagerQualityCodesXlsxHandler(xml.sax.handler.ContentHandler):
  """
  Sax parser callback handler.
  """
  def __init__(self):
    self.string_dictionary = None
    self.quality_code_dictionary = {}
    # Clear cursors and buffers.
    self.reset()
  
  
  def reset(self):
    self.clear_buffers()
    self.clear_cursors()
    
    
  def clear_buffers(self):
    self.buf_quality_code = ''
    self.buf_desc_string_code = ''
    
    
  def clear_cursors(self):
    self.n_rows = 0
    self.n_cols = 0
  
  
  def startElement(self, name, attrs):
    if name == "sheetData":
      # Clear cursors and buffers.
      self.reset()
    if name == "row":
      self.n_cols = 0
      self.n_rows += 1
    if name == "c":
      self.n_cols += 1
    del name
    del attrs
    
    
  def characters(self, content):
    if self.n_rows > 1:
      if self.n_cols == 1:
	#c1: actual quality code
	self.buf_quality_code += content
      if self.n_cols == 2:
	#c2: numerical code for the string description
	self.buf_desc_string_code += content
    del content
    
    
  def endElement(self, name):
    if name == "row":
      if self.n_rows > 1:
	self.quality_code_dictionary[self.buf_quality_code] = self.string_dictionary[int(self.buf_desc_string_code)]
	self.clear_buffers()
  

class DpipweSourceXlsxHandler(xml.sax.handler.ContentHandler):
  """
  Sax parser callback handler.
  """
  def __init__(self):
    import re
    self.datetime_format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss")
    self.datetime_format = self.datetime_format.withZone(DateTimeZone.UTC)
    self.uom_pattern = re.compile("^(\\w+)\\s+\\((.*)\\)$")
    self.string_dictionary = None
    self.quality_code_dictionary = {}
    self.messages = list()
    # Clear cursors and buffers.
    self.reset()
  
  
  def reset(self):
    self.sensor_id = None
    self.uom = None
    self.clear_buffers()
    self.clear_cursors()
    del self.messages[:]
    
    
  def clear_buffers(self):
    self.buf_date = ''
    self.buf_time = ''
    self.buf_value = ''
    self.buf_quality = ''
    self.buf_sensor_uom = ''
    self.quality_code = ''
    self.quality_code_description = ''
    
    
  def clear_cursors(self):
    self.n_rows = 0
    self.n_cols = 0
  
  
  def startElement(self, name, attrs):
    if name == "sheetData":
      # Clear cursors and buffers.
      self.reset()
    if name == "row":
      self.n_cols = 0
      self.n_rows += 1
    if name == "c":
      self.n_cols += 1
    del name
    del attrs
    
    
  def characters(self, content):
    if self.n_rows == 1:
      if self.n_cols == 3:
	self.buf_sensor_uom += content
    if self.n_rows > 2:
      if self.n_cols == 1:
	#c1: date = days since 1899-12-31
	self.buf_date += content
      if self.n_cols == 2:
	#c2: time = fraction of day. e.g. 12 noon = 0.5
	self.buf_time += content
      if self.n_cols == 3:
	#c3: obs = n uom
	self.buf_value += content
      if self.n_cols == 4:
	#c4: quality = string ^(Q\\d+)$
	self.buf_quality += content
    del content
    
    
  def endElement(self, name):
    if name == "row":
      if self.n_rows == 1:
	uom_string = self.string_dictionary[int(self.buf_sensor_uom)]
	match = self.uom_pattern.match(uom_string)
	if match:
	  self.sensor_id = match.group(1).lower()
	  if "deg c" in match.group(2).lower():
	    self.uom = "degrees C"
	  else:
	    self.uom = match.group(2)
      
      if self.n_rows > 2 and self.sensor_id is not None:
	d = xl_date().datetime_from_days_since_excel_epoch(int(self.buf_date))
	t = xl_date().time_from_day_fraction(float(self.buf_time))
	dt = d.strftime("%Y-%m-%d") + 'T' + t.strftime("%H:%M:%S")
	val = float(self.buf_value)
	
	quality = self.string_dictionary[int(self.buf_quality)]

	# Create sample and message 
	s = Sample()
	s.time = self.datetime_format.parseDateTime(dt)
	s.value = val
	s.sensorId = "dpipwe.new_norfolk." + self.sensor_id
	s.tags.put("uom", self.uom)
	if quality != '':
	  s.tags.put("quality_code", quality)
	  if quality[1:] in self.quality_code_dictionary:
	    s.tags.put("quality_description", self.quality_code_dictionary[quality[1:]])
	message = Message(s.sensorId, s)
	self.messages.append(message)
      self.clear_buffers()
      
    # Be nice to the dispatcher, batch up the messages. 
    if len(self.messages) > 19 or name == "sheetData":
      source.dispatch(self.messages)
      del self.messages[:]


